setwd("~/Dropbox (Facebook)/Deep Cooperative Game Theory/cdm_experiments/nba")
require(ggplot2)
require(dplyr)
require(reshape2)
fo_preds <- read.csv("first_order_nba_prediction.csv", header=FALSE)
fo_preds$model <- 'first_order'
so_preds <- read.csv("cdm_nba_prediction.csv", header=FALSE)
so_preds$model <- 'second_order'
ds_preds <- read.csv("deepset_nba_prediction.csv", header=FALSE)
ds_preds$model <- 'deepset'
preds <- rbind(fo_preds, so_preds, ds_preds)
names(preds) <- c('set', 'team', 'truth', 'pred', 'model')
preds$nll <- preds$truth*log(preds$pred) + (1-preds$truth)*(log(1-preds$pred))
preds %>%
mutate(pred_outcome = ifelse(pred>.5, 1, 0)) %>%
mutate(correct = ifelse(pred_outcome == truth, 1, 0)) %>%
group_by(set, model) %>%
summarise(m = mean(correct),
sd = sd(correct) / sqrt(n()),
mnll = mean(nll))
setwd("~/Dropbox (Facebook)/Deep Cooperative Game Theory/Experiments/nba")
require(ggplot2)
require(dplyr)
require(reshape2)
fo_preds <- read.csv("first_order_nba_prediction.csv", header=FALSE)
fo_preds$model <- 'first_order'
so_preds <- read.csv("cdm_nba_prediction.csv", header=FALSE)
so_preds$model <- 'second_order'
ds_preds <- read.csv("deepset_nba_prediction.csv", header=FALSE)
ds_preds$model <- 'deepset'
preds <- rbind(fo_preds, so_preds, ds_preds)
names(preds) <- c('set', 'team', 'truth', 'pred', 'model')
preds$nll <- preds$truth*log(preds$pred) + (1-preds$truth)*(log(1-preds$pred))
preds %>%
mutate(pred_outcome = ifelse(pred>.5, 1, 0)) %>%
mutate(correct = ifelse(pred_outcome == truth, 1, 0)) %>%
group_by(set, model) %>%
summarise(m = mean(correct),
sd = sd(correct) / sqrt(n()),
mnll = mean(nll))
nba_stats <- read.csv("nba_stats.csv")
deepset_stats <- read.csv("deepset_nba_stats.csv")
nba_stats$shapley <- scale(nba_stats$shapley)
nba_stats$average_salary <- log(nba_stats$average_salary)
cor(nba_stats %>% select(win_share, vorp, average_salary, shapley, bias))
stats_long <- melt(nba_stats, id=c('name', 'team', 'shapley'))
stats_long <- stats_long %>% filter(variable != 'bias')
stats_long$Var2 <- 'VORP'
stats_long$Var2[stats_long$variable == 'average_salary'] <- 'Log(Salary)'
stats_long$Var2[stats_long$variable == 'win_share'] <- 'Win Share'
setwd("~/Dropbox (Facebook)/Deep Cooperative Game Theory/paper/")
pdf("nba_plots.pdf", width=6, height=4)
ggplot(data=stats_long, aes(x=shapley, y=value)) +
geom_point() +
geom_smooth(method='lm') +
facet_wrap(~Var2, scale='free') +
xlab("Normalized Shapley Value") +
ylab("NBA Metric") +
theme_bw() +
theme(text=element_text(size=12))
dev.off()
setwd("~/Dropbox (Facebook)/Deep Cooperative Game Theory/Experiments/nba")
all_star_scores <- read.csv("all_star.csv", header=FALSE)
all_starb_scores <- read.csv("all_star_backup.csv", header=FALSE)
random_scores <- melt(read.csv("random_team1k.csv", header=FALSE))
setwd("~/Dropbox (Facebook)/Deep Cooperative Game Theory/paper/")
pdf("nba_allstar_hist.pdf", width=6, height=4)
qplot(x=random_scores$value, geom='density', fill='Random Team') +
geom_vline(aes(xintercept=all_star_scores$V7, colour='All Star Team'), fill='All Star Team', size=1) +
geom_vline(aes(xintercept=all_starb_scores$V7, colour='All Star Backups'), fill='All Star Backups', size=1) +
xlab("CGA Predicted Score") +
ylab("Density") +
theme_bw() +
theme(text=element_text(size=12)) +
theme(legend.title=element_blank(), legend.position='bottom') +
scale_color_manual(name = "statistics", values = c(`All Star Team` = "blue", `All Star Backups` = "green"))
dev.off()
setwd("~/Dropbox (Facebook)/Deep Cooperative Game Theory/Experiments/nba")
require(ggplot2)
require(dplyr)
require(reshape2)
fo_preds <- read.csv("first_order_nba_prediction.csv", header=FALSE)
fo_preds$model <- 'first_order'
so_preds <- read.csv("cdm_nba_prediction.csv", header=FALSE)
so_preds$model <- 'second_order'
ds_preds <- read.csv("deepset_nba_prediction.csv", header=FALSE)
ds_preds$model <- 'deepset'
preds <- rbind(fo_preds, so_preds, ds_preds)
names(preds) <- c('set', 'team', 'truth', 'pred', 'model')
preds$nll <- preds$truth*log(preds$pred) + (1-preds$truth)*(log(1-preds$pred))
preds %>%
mutate(pred_outcome = ifelse(pred>.5, 1, 0)) %>%
mutate(correct = ifelse(pred_outcome == truth, 1, 0)) %>%
group_by(set, model) %>%
summarise(m = mean(correct),
sd = sd(correct) / sqrt(n()),
mnll = mean(nll))
setwd("~/Dropbox (Facebook)/Deep Cooperative Game Theory/Experiments/nba")
require(ggplot2)
require(dplyr)
require(reshape2)
fo_preds <- read.csv("first_order_nba_prediction.csv", header=FALSE)
fo_preds$model <- 'first_order'
so_preds <- read.csv("second_order_nba_prediction.csv", header=FALSE)
so_preds$model <- 'second_order'
ds_preds <- read.csv("deepset_nba_prediction.csv", header=FALSE)
ds_preds$model <- 'deepset'
preds <- rbind(fo_preds, so_preds, ds_preds)
names(preds) <- c('set', 'team', 'truth', 'pred', 'model')
preds$nll <- preds$truth*log(preds$pred) + (1-preds$truth)*(log(1-preds$pred))
preds %>%
mutate(pred_outcome = ifelse(pred>.5, 1, 0)) %>%
mutate(correct = ifelse(pred_outcome == truth, 1, 0)) %>%
group_by(set, model) %>%
summarise(m = mean(correct),
sd = sd(correct) / sqrt(n()),
mnll = mean(nll))
setwd("~/Dropbox (Facebook)/Deep Cooperative Game Theory/Experiments/nba")
require(ggplot2)
require(dplyr)
require(reshape2)
fo_preds <- read.csv("first_order_nba_prediction.csv", header=FALSE)
fo_preds$model <- 'first_order'
so_preds <- read.csv("second_order_nba_prediction.csv", header=FALSE)
so_preds$model <- 'second_order'
ds_preds <- read.csv("deepset_nba_prediction.csv", header=FALSE)
ds_preds$model <- 'deepset'
preds <- rbind(fo_preds, so_preds, ds_preds)
names(preds) <- c('set', 'team', 'truth', 'pred', 'model')
preds$nll <- preds$truth*log(preds$pred) + (1-preds$truth)*(log(1-preds$pred))
preds %>%
mutate(pred_outcome = ifelse(pred>.5, 1, 0)) %>%
mutate(correct = ifelse(pred_outcome == truth, 1, 0)) %>%
group_by(set, model) %>%
summarise(m = mean(correct),
sd = sd(correct) / sqrt(n()),
mnll = mean(nll))
nba_stats <- read.csv("nba_stats.csv")
deepset_stats <- read.csv("deepset_nba_stats.csv")
nba_stats$shapley <- scale(nba_stats$shapley)
nba_stats$average_salary <- log(nba_stats$average_salary)
cor(nba_stats %>% select(win_share, vorp, average_salary, shapley, bias))
stats_long <- melt(nba_stats, id=c('name', 'team', 'shapley'))
stats_long <- stats_long %>% filter(variable != 'bias')
stats_long$Var2 <- 'VORP'
stats_long$Var2[stats_long$variable == 'average_salary'] <- 'Log(Salary)'
stats_long$Var2[stats_long$variable == 'win_share'] <- 'Win Share'
setwd("~/Dropbox (Facebook)/Deep Cooperative Game Theory/paper/")
pdf("nba_plots.pdf", width=6, height=4)
ggplot(data=stats_long, aes(x=shapley, y=value)) +
geom_point() +
geom_smooth(method='lm') +
facet_wrap(~Var2, scale='free') +
xlab("Normalized Shapley Value") +
ylab("NBA Metric") +
theme_bw() +
theme(text=element_text(size=12))
dev.off()
setwd("~/Dropbox (Facebook)/Deep Cooperative Game Theory/Experiments/nba")
all_star_scores <- read.csv("all_star.csv", header=FALSE)
all_starb_scores <- read.csv("all_star_backup.csv", header=FALSE)
random_scores <- melt(read.csv("random_team1k.csv", header=FALSE))
setwd("~/Dropbox (Facebook)/Deep Cooperative Game Theory/paper/")
pdf("nba_allstar_hist.pdf", width=6, height=4)
qplot(x=random_scores$value, geom='density', fill='Random Team') +
geom_vline(aes(xintercept=all_star_scores$V7, colour='All Star Team'), fill='All Star Team', size=1) +
geom_vline(aes(xintercept=all_starb_scores$V7, colour='All Star Backups'), fill='All Star Backups', size=1) +
xlab("CGA Predicted Score") +
ylab("Density") +
theme_bw() +
theme(text=element_text(size=12)) +
theme(legend.title=element_blank(), legend.position='bottom') +
scale_color_manual(name = "statistics", values = c(`All Star Team` = "blue", `All Star Backups` = "green"))
dev.off()
setwd("~/Dropbox (Facebook)/Deep Cooperative Game Theory/Experiments/openai_particle")
require(ggplot2)
require(dplyr)
require(reshape2)
fo_preds <- read.csv("first_order_spread_prediction.csv", header=FALSE)
fo_preds$model <- 'first_order'
so_preds <- read.csv("second_order_spread_prediction.csv", header=FALSE)
so_preds$model <- 'second_order'
ds_preds <- read.csv("deepset_spread_prediction.csv", header=FALSE)
ds_preds$model <- 'deepset'
preds <- rbind(fo_preds, so_preds, ds_preds)
names(preds) <- c('set', 'team', 'truth', 'pred', 'model')
preds %>%
mutate(error = (truth - pred)^2) %>%
group_by(set, model) %>%
summarise(m = mean(error),
sd = sd(error) / sqrt(n()))
select_cga2 <- read.csv("select_best_pair.csv", header=FALSE)
select_cga2$model <- '2CGA'
names(select_cga2) <- c('target', 'teamcga', 'teamtrue', 'model_val', 'best_val', 'random_val', 'model')
select_cga1 <- read.csv("select_best_pair_lr.csv", header=FALSE)
select_cga1$model <- '1CGA'
names(select_cga1) <- c('target', 'teamcga', 'teamtrue', 'model_val', 'best_val', 'random_val', 'model')
select_deepset <- read.csv("select_best_pair_deepset.csv", header=FALSE)
select_deepset$model <- 'Deep Set'
names(select_deepset) <- c('target', 'teamcga', 'teamtrue', 'model_val', 'best_val', 'random_val', 'model')
select <- rbind(select_cga2, select_cga1, select_deepset)
names(select) <- c('target', 'teamcga', 'teamtrue', 'model_val', 'best_val', 'random_val', 'model')
select$ratio <- select$model_val / select$best_val
setwd("~/Dropbox (Facebook)/Deep Cooperative Game Theory/paper/")
pdf("particle_pairs.pdf", width=6, height=4)
qplot(data=select, x=ratio) +
facet_wrap(~model) +
scale_x_continuous(limits=c(0,1.05)) +
theme_bw() +
xlab("Percentage of Best Team Score") +
ylab("Count") +
theme(text=element_text(size=12))
dev.off()
preds %>%
mutate(error = (truth - pred)^2) %>%
group_by(set, model) %>%
summarise(m = mean(error),
sd = sd(error) / sqrt(n()))
setwd("~/Dropbox (Facebook)/Deep Cooperative Game Theory/cdm_experiments/openai_particle")
require(ggplot2)
require(dplyr)
require(reshape2)
fo_preds <- read.csv("first_order_spread_prediction.csv", header=FALSE)
fo_preds$model <- 'first_order'
so_preds <- read.csv("cdm_spread_prediction.csv", header=FALSE)
so_preds$model <- 'second_order'
ds_preds <- read.csv("deepset_spread_prediction.csv", header=FALSE)
ds_preds$model <- 'deepset'
preds <- rbind(fo_preds, so_preds, ds_preds)
names(preds) <- c('set', 'team', 'truth', 'pred', 'model')
preds %>%
mutate(error = (truth - pred)^2) %>%
group_by(set, model) %>%
summarise(m = mean(error),
sd = sd(error) / sqrt(n()))
setwd("~/Dropbox (Facebook)/Deep Cooperative Game Theory/cdm_experiments/openai_particle")
require(ggplot2)
require(dplyr)
require(reshape2)
fo_preds <- read.csv("first_order_spread_prediction.csv", header=FALSE)
fo_preds$model <- 'first_order'
so_preds <- read.csv("cdm_spread_prediction.csv", header=FALSE)
so_preds$model <- 'second_order'
ds_preds <- read.csv("deepset_spread_prediction.csv", header=FALSE)
ds_preds$model <- 'deepset'
preds <- rbind(fo_preds, so_preds, ds_preds)
names(preds) <- c('set', 'team', 'truth', 'pred', 'model')
preds %>%
mutate(error = (truth - pred)^2) %>%
group_by(set, model) %>%
summarise(m = mean(error),
sd = sd(error) / sqrt(n()))
select_cga2 <- read.csv("select_best_pair.csv", header=FALSE)
select_cga2$model <- '2CGA'
names(select_cga2) <- c('target', 'teamcga', 'teamtrue', 'model_val', 'best_val', 'random_val', 'model')
select_cga1 <- read.csv("select_best_pair_lr.csv", header=FALSE)
select_cga1$model <- '1CGA'
names(select_cga1) <- c('target', 'teamcga', 'teamtrue', 'model_val', 'best_val', 'random_val', 'model')
select_deepset <- read.csv("select_best_pair_deepset.csv", header=FALSE)
select_deepset$model <- 'Deep Set'
names(select_deepset) <- c('target', 'teamcga', 'teamtrue', 'model_val', 'best_val', 'random_val', 'model')
select <- rbind(select_cga2, select_cga1, select_deepset)
names(select) <- c('target', 'teamcga', 'teamtrue', 'model_val', 'best_val', 'random_val', 'model')
select$ratio <- select$model_val / select$best_val
setwd("~/Dropbox (Facebook)/Deep Cooperative Game Theory/paper/")
pdf("particle_pairs.pdf", width=6, height=4)
qplot(data=select, x=ratio) +
facet_wrap(~model) +
scale_x_continuous(limits=c(0,1.05)) +
theme_bw() +
xlab("Percentage of Best Team Score") +
ylab("Count") +
theme(text=element_text(size=12))
dev.off()
setwd("~/Dropbox (Facebook)/Deep Cooperative Game Theory/cdm_experiments/nba")
require(ggplot2)
require(dplyr)
require(reshape2)
fo_preds <- read.csv("first_order_nba_prediction.csv", header=FALSE)
fo_preds$model <- 'first_order'
so_preds <- read.csv("cdm_nba_prediction.csv", header=FALSE)
so_preds$model <- 'second_order'
ds_preds <- read.csv("deepset_nba_prediction.csv", header=FALSE)
ds_preds$model <- 'deepset'
preds <- rbind(fo_preds, so_preds, ds_preds)
names(preds) <- c('set', 'team', 'truth', 'pred', 'model')
preds$nll <- preds$truth*log(preds$pred) + (1-preds$truth)*(log(1-preds$pred))
preds %>%
mutate(pred_outcome = ifelse(pred>.5, 1, 0)) %>%
mutate(correct = ifelse(pred_outcome == truth, 1, 0)) %>%
group_by(set, model) %>%
summarise(m = mean(correct),
sd = sd(correct) / sqrt(n()),
mnll = mean(nll))
nba_stats <- read.csv("nba_stats.csv")
deepset_stats <- read.csv("deepset_nba_stats.csv")
nba_stats$shapley <- scale(nba_stats$shapley)
nba_stats$average_salary <- log(nba_stats$average_salary)
cor(nba_stats %>% select(win_share, vorp, average_salary, shapley, bias))
stats_long <- melt(nba_stats, id=c('name', 'team', 'shapley'))
stats_long <- stats_long %>% filter(variable != 'bias')
stats_long$Var2 <- 'VORP'
stats_long$Var2[stats_long$variable == 'average_salary'] <- 'Log(Salary)'
stats_long$Var2[stats_long$variable == 'win_share'] <- 'Win Share'
setwd("~/Dropbox (Facebook)/Deep Cooperative Game Theory/paper/")
pdf("nba_plots.pdf", width=6, height=4)
ggplot(data=stats_long, aes(x=shapley, y=value)) +
geom_point() +
geom_smooth(method='lm') +
facet_wrap(~Var2, scale='free') +
xlab("Normalized Shapley Value") +
ylab("NBA Metric") +
theme_bw() +
theme(text=element_text(size=12))
dev.off()
setwd("~/Dropbox (Facebook)/Deep Cooperative Game Theory/Experiments/nba")
all_star_scores <- read.csv("all_star.csv", header=FALSE)
all_starb_scores <- read.csv("all_star_backup.csv", header=FALSE)
random_scores <- melt(read.csv("random_team1k.csv", header=FALSE))
setwd("~/Dropbox (Facebook)/Deep Cooperative Game Theory/paper/")
pdf("nba_allstar_hist.pdf", width=6, height=4)
qplot(x=random_scores$value, geom='density', fill='Random Team') +
geom_vline(aes(xintercept=all_star_scores$V7, colour='All Star Team'), fill='All Star Team', size=1) +
geom_vline(aes(xintercept=all_starb_scores$V7, colour='All Star Backups'), fill='All Star Backups', size=1) +
xlab("CGA Predicted Score") +
ylab("Density") +
theme_bw() +
theme(text=element_text(size=12)) +
theme(legend.title=element_blank(), legend.position='bottom') +
scale_color_manual(name = "statistics", values = c(`All Star Team` = "blue", `All Star Backups` = "green"))
dev.off()
setwd("~/Dropbox (Facebook)/Deep Cooperative Game Theory/cdm_experiments/nba")
require(ggplot2)
require(dplyr)
require(reshape2)
fo_preds <- read.csv("first_order_nba_prediction.csv", header=FALSE)
fo_preds$model <- 'first_order'
so_preds <- read.csv("cdm_nba_prediction.csv", header=FALSE)
so_preds$model <- 'second_order'
ds_preds <- read.csv("deepset_nba_prediction.csv", header=FALSE)
ds_preds$model <- 'deepset'
preds <- rbind(fo_preds, so_preds, ds_preds)
names(preds) <- c('set', 'team', 'truth', 'pred', 'model')
preds$nll <- preds$truth*log(preds$pred) + (1-preds$truth)*(log(1-preds$pred))
preds %>%
mutate(pred_outcome = ifelse(pred>.5, 1, 0)) %>%
mutate(correct = ifelse(pred_outcome == truth, 1, 0)) %>%
group_by(set, model) %>%
summarise(m = mean(correct),
sd = sd(correct) / sqrt(n()),
mnll = mean(nll))
head(preds)
qplot(data=preds, x=pred, y=truth)
qplot(data=preds, x=pred, fill=truth, geom='density')
qplot(data=preds, x=pred, fill=truth, geom='density', group=truth)
qplot(data=preds %>% filter(set == 'test'), x=pred, fill=truth, geom='density', group=truth)
qplot(data=preds %>% filter(set == 'test'), x=pred, fill=truth, geom='density', group=truth, alpha=.5\)
qplot(data=preds %>% filter(set == 'test'), x=pred, fill=truth, geom='density', group=truth, alpha=.5)
